{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM-CNN Text Classification with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Loaded Parameters:\n",
      " 40000 30 0.2 50 \n",
      " dataset/glove/glove.twitter.27B.50d.txt\n",
      "[i] Importing Modules...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Using Keras version 2.1.4\n",
      "[i] Finished Importing Modules\n",
      "[i] . Reading from csv file...Done!\n",
      "[i] Found 30692 unique tokens.\n",
      "[+] Shape of data tensor: (40000, 30)\n",
      "[+] Shape of label tensor: (40000, 4)\n",
      "[+] Number of entries in each category:\n",
      "[+] Training:\n",
      " [ 7690. 10981. 12192.  1137.]\n",
      "[+] Validation:\n",
      " [1954. 2643. 3107.  296.]\n",
      "[i] Loading GloVe from: dataset/glove/glove.twitter.27B.50d.txt ...Done.\n",
      "[+] Proceeding with Embedding Matrix...Completed!\n",
      "Finished running setup.\n"
     ]
    }
   ],
   "source": [
    "%run Setup.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load embedding matrix into an `Embedding` layer. Toggle `trainable=False` to prevent the weights from being updated during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN Structure\n",
    "[Reference](https://github.com/richliao/textClassifier), [LTSM](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "l_lstm1 = Bidirectional(LSTM(12,dropout=0.4,recurrent_dropout=0.4,return_sequences=True))(embedded_sequences)\n",
    "l_cov1= Conv1D(50, 3, activation='relu',kernel_regularizer=regularizers.l2(0.02))(l_lstm1)\n",
    "l_pool1 = MaxPooling1D(2)(l_cov1)\n",
    "l_drop1 = Dropout(0.4)(l_pool1)\n",
    "#l_cov2 = Conv1D(32, 3, activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_drop1)\n",
    "#l_pool2 = MaxPooling1D(2)(l_cov2)\n",
    "#l_drop2 = Dropout(0.3)(l_pool2)\n",
    "l_flat = Flatten()(l_drop1)\n",
    "l_dense = Dense(25, activation='relu')(l_flat)\n",
    "preds = Dense(4, activation='softmax')(l_dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lr_metric(optimizer):\n",
    "    def lr(y_true, y_pred):\n",
    "        return optimizer.lr\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = Model(sequence_input, preds)\n",
    "adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)\n",
    "lr_metric = get_lr_metric(adadelta)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=adadelta,\n",
    "              metrics=['acc', lr_metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def step_cyclic(epoch):\n",
    "    try:\n",
    "        l_r, decay = 1.0, 0.00001\n",
    "        if epoch%33==0:multiplier = 10\n",
    "        else:multiplier = 1\n",
    "        rate = float(multiplier * l_r * 1/(1 + decay * epoch))\n",
    "        print(\"Epoch\",epoch+1,\"- learning_rate\",rate)\n",
    "        return rate\n",
    "    except Exception as e:\n",
    "        print(\"Error in lr_schedule:\",str(e))\n",
    "        return float(1.0)\n",
    "    \n",
    "def initial_boost(epoch):\n",
    "    if epoch==0: return float(6.0)\n",
    "    else: return float(1.0)\n",
    "        \n",
    "tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=5, batch_size=16, write_grads=True , write_graph=True)\n",
    "model_checkpoints = callbacks.ModelCheckpoint(\"checkpoint.h5\", monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)\n",
    "#lr_schedule = callbacks.LearningRateScheduler(step_cyclic)\n",
    "lr_schedule = callbacks.LearningRateScheduler(step_cyclic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 30)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 30, 50)            1534650   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 30, 24)            6048      \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 28, 50)            3650      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 14, 50)            0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 14, 50)            0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 700)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 25)                17525     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4)                 104       \n",
      "=================================================================\n",
      "Total params: 1,561,977\n",
      "Trainable params: 1,561,977\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Training Progress:\n",
      "Train on 32000 samples, validate on 8000 samples\n",
      "Epoch 1/200\n",
      "Epoch 1 - learning_rate 10.0\n",
      "32000/32000 [==============================] - 32s 1ms/step - loss: 1.2068 - acc: 0.4056 - lr: 10.0000 - val_loss: 1.1266 - val_acc: 0.5095 - val_lr: 10.0000\n",
      "Epoch 2/200\n",
      "Epoch 2 - learning_rate 0.9999900000999989\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 1.1136 - acc: 0.4947 - lr: 1.0000 - val_loss: 1.0735 - val_acc: 0.5265 - val_lr: 1.0000\n",
      "Epoch 3/200\n",
      "Epoch 3 - learning_rate 0.9999800003999921\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 1.0984 - acc: 0.5066 - lr: 1.0000 - val_loss: 1.0633 - val_acc: 0.5381 - val_lr: 1.0000\n",
      "Epoch 4/200\n",
      "Epoch 4 - learning_rate 0.999970000899973\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 1.0869 - acc: 0.5164 - lr: 1.0000 - val_loss: 1.0500 - val_acc: 0.5473 - val_lr: 1.0000\n",
      "Epoch 5/200\n",
      "Epoch 5 - learning_rate 0.9999600015999359\n",
      "32000/32000 [==============================] - 29s 909us/step - loss: 1.0769 - acc: 0.5248 - lr: 1.0000 - val_loss: 1.0410 - val_acc: 0.5536 - val_lr: 1.0000\n",
      "Epoch 6/200\n",
      "Epoch 6 - learning_rate 0.9999500024998749\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 1.0683 - acc: 0.5311 - lr: 0.9999 - val_loss: 1.0344 - val_acc: 0.5591 - val_lr: 0.9999\n",
      "Epoch 7/200\n",
      "Epoch 7 - learning_rate 0.9999400035997841\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 1.0576 - acc: 0.5428 - lr: 0.9999 - val_loss: 1.0275 - val_acc: 0.5646 - val_lr: 0.9999\n",
      "Epoch 8/200\n",
      "Epoch 8 - learning_rate 0.999930004899657\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 1.0514 - acc: 0.5486 - lr: 0.9999 - val_loss: 1.0233 - val_acc: 0.5720 - val_lr: 0.9999\n",
      "Epoch 9/200\n",
      "Epoch 9 - learning_rate 0.999920006399488\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 1.0458 - acc: 0.5496 - lr: 0.9999 - val_loss: 1.0136 - val_acc: 0.5763 - val_lr: 0.9999\n",
      "Epoch 10/200\n",
      "Epoch 10 - learning_rate 0.9999100080992711\n",
      "32000/32000 [==============================] - 29s 909us/step - loss: 1.0405 - acc: 0.5551 - lr: 0.9999 - val_loss: 1.0163 - val_acc: 0.5766 - val_lr: 0.9999\n",
      "Epoch 11/200\n",
      "Epoch 11 - learning_rate 0.9999000099990001\n",
      "32000/32000 [==============================] - 29s 906us/step - loss: 1.0352 - acc: 0.5525 - lr: 0.9999 - val_loss: 1.0029 - val_acc: 0.5833 - val_lr: 0.9999\n",
      "Epoch 12/200\n",
      "Epoch 12 - learning_rate 0.9998900120986691\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 1.0251 - acc: 0.5633 - lr: 0.9999 - val_loss: 1.0007 - val_acc: 0.5837 - val_lr: 0.9999\n",
      "Epoch 13/200\n",
      "Epoch 13 - learning_rate 0.9998800143982723\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 1.0206 - acc: 0.5668 - lr: 0.9999 - val_loss: 0.9992 - val_acc: 0.5853 - val_lr: 0.9999\n",
      "Epoch 14/200\n",
      "Epoch 14 - learning_rate 0.9998700168978033\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 1.0199 - acc: 0.5656 - lr: 0.9999 - val_loss: 1.0026 - val_acc: 0.5834 - val_lr: 0.9999\n",
      "Epoch 15/200\n",
      "Epoch 15 - learning_rate 0.9998600195972563\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 1.0150 - acc: 0.5724 - lr: 0.9999 - val_loss: 0.9989 - val_acc: 0.5843 - val_lr: 0.9999\n",
      "Epoch 16/200\n",
      "Epoch 16 - learning_rate 0.9998500224966254\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 1.0124 - acc: 0.5719 - lr: 0.9999 - val_loss: 0.9890 - val_acc: 0.5857 - val_lr: 0.9999\n",
      "Epoch 17/200\n",
      "Epoch 17 - learning_rate 0.9998400255959047\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 1.0090 - acc: 0.5769 - lr: 0.9998 - val_loss: 0.9927 - val_acc: 0.5894 - val_lr: 0.9998\n",
      "Epoch 18/200\n",
      "Epoch 18 - learning_rate 0.9998300288950879\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 1.0083 - acc: 0.5735 - lr: 0.9998 - val_loss: 0.9901 - val_acc: 0.5900 - val_lr: 0.9998\n",
      "Epoch 19/200\n",
      "Epoch 19 - learning_rate 0.999820032394169\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 1.0024 - acc: 0.5798 - lr: 0.9998 - val_loss: 0.9828 - val_acc: 0.5899 - val_lr: 0.9998\n",
      "Epoch 20/200\n",
      "Epoch 20 - learning_rate 0.9998100360931423\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 0.9984 - acc: 0.5809 - lr: 0.9998 - val_loss: 0.9851 - val_acc: 0.5906 - val_lr: 0.9998\n",
      "Epoch 21/200\n",
      "Epoch 21 - learning_rate 0.9998000399920016\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 0.9972 - acc: 0.5814 - lr: 0.9998 - val_loss: 0.9820 - val_acc: 0.5903 - val_lr: 0.9998\n",
      "Epoch 22/200\n",
      "Epoch 22 - learning_rate 0.9997900440907409\n",
      "32000/32000 [==============================] - 32s 985us/step - loss: 0.9951 - acc: 0.5836 - lr: 0.9998 - val_loss: 0.9815 - val_acc: 0.5913 - val_lr: 0.9998\n",
      "Epoch 23/200\n",
      "Epoch 23 - learning_rate 0.9997800483893542\n",
      "32000/32000 [==============================] - 29s 895us/step - loss: 0.9929 - acc: 0.5847 - lr: 0.9998 - val_loss: 0.9874 - val_acc: 0.5871 - val_lr: 0.9998\n",
      "Epoch 24/200\n",
      "Epoch 24 - learning_rate 0.9997700528878358\n",
      "32000/32000 [==============================] - 29s 895us/step - loss: 0.9909 - acc: 0.5843 - lr: 0.9998 - val_loss: 0.9814 - val_acc: 0.5939 - val_lr: 0.9998\n",
      "Epoch 25/200\n",
      "Epoch 25 - learning_rate 0.9997600575861793\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 0.9883 - acc: 0.5889 - lr: 0.9998 - val_loss: 0.9849 - val_acc: 0.5934 - val_lr: 0.9998\n",
      "Epoch 26/200\n",
      "Epoch 26 - learning_rate 0.9997500624843788\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.9856 - acc: 0.5903 - lr: 0.9998 - val_loss: 0.9777 - val_acc: 0.5920 - val_lr: 0.9998\n",
      "Epoch 27/200\n",
      "Epoch 27 - learning_rate 0.9997400675824286\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.9842 - acc: 0.5903 - lr: 0.9997 - val_loss: 0.9843 - val_acc: 0.5906 - val_lr: 0.9997\n",
      "Epoch 28/200\n",
      "Epoch 28 - learning_rate 0.9997300728803223\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9822 - acc: 0.5941 - lr: 0.9997 - val_loss: 0.9804 - val_acc: 0.5873 - val_lr: 0.9997\n",
      "Epoch 29/200\n",
      "Epoch 29 - learning_rate 0.9997200783780541\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.9791 - acc: 0.5930 - lr: 0.9997 - val_loss: 0.9800 - val_acc: 0.5866 - val_lr: 0.9997\n",
      "Epoch 30/200\n",
      "Epoch 30 - learning_rate 0.9997100840756181\n",
      "32000/32000 [==============================] - 31s 965us/step - loss: 0.9767 - acc: 0.5958 - lr: 0.9997 - val_loss: 0.9894 - val_acc: 0.5887 - val_lr: 0.9997\n",
      "Epoch 31/200\n",
      "Epoch 31 - learning_rate 0.9997000899730081\n",
      "32000/32000 [==============================] - 31s 964us/step - loss: 0.9767 - acc: 0.5960 - lr: 0.9997 - val_loss: 0.9770 - val_acc: 0.5910 - val_lr: 0.9997\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32/200\n",
      "Epoch 32 - learning_rate 0.9996900960702182\n",
      "32000/32000 [==============================] - 28s 880us/step - loss: 0.9740 - acc: 0.5954 - lr: 0.9997 - val_loss: 0.9788 - val_acc: 0.5907 - val_lr: 0.9997\n",
      "Epoch 33/200\n",
      "Epoch 33 - learning_rate 0.9996801023672424\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 0.9743 - acc: 0.5981 - lr: 0.9997 - val_loss: 0.9853 - val_acc: 0.5941 - val_lr: 0.9997\n",
      "Epoch 34/200\n",
      "Epoch 34 - learning_rate 9.99670108864075\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 1.0285 - acc: 0.5725 - lr: 9.9967 - val_loss: 1.0265 - val_acc: 0.5725 - val_lr: 9.9967\n",
      "Epoch 35/200\n",
      "Epoch 35 - learning_rate 0.9996601155607093\n",
      "32000/32000 [==============================] - 28s 884us/step - loss: 0.9859 - acc: 0.5980 - lr: 0.9997 - val_loss: 0.9952 - val_acc: 0.5883 - val_lr: 0.9997\n",
      "Epoch 36/200\n",
      "Epoch 36 - learning_rate 0.9996501224571399\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.9749 - acc: 0.6001 - lr: 0.9997 - val_loss: 0.9837 - val_acc: 0.5871 - val_lr: 0.9997\n",
      "Epoch 37/200\n",
      "Epoch 37 - learning_rate 0.9996401295533609\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9722 - acc: 0.6024 - lr: 0.9996 - val_loss: 0.9934 - val_acc: 0.5916 - val_lr: 0.9996\n",
      "Epoch 38/200\n",
      "Epoch 38 - learning_rate 0.9996301368493657\n",
      "32000/32000 [==============================] - 29s 895us/step - loss: 0.9662 - acc: 0.6043 - lr: 0.9996 - val_loss: 0.9913 - val_acc: 0.5885 - val_lr: 0.9996\n",
      "Epoch 39/200\n",
      "Epoch 39 - learning_rate 0.9996201443451488\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 0.9649 - acc: 0.6040 - lr: 0.9996 - val_loss: 0.9845 - val_acc: 0.5876 - val_lr: 0.9996\n",
      "Epoch 40/200\n",
      "Epoch 40 - learning_rate 0.9996101520407042\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9630 - acc: 0.6057 - lr: 0.9996 - val_loss: 0.9786 - val_acc: 0.5913 - val_lr: 0.9996\n",
      "Epoch 41/200\n",
      "Epoch 41 - learning_rate 0.9996001599360256\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 0.9611 - acc: 0.6060 - lr: 0.9996 - val_loss: 0.9902 - val_acc: 0.5900 - val_lr: 0.9996\n",
      "Epoch 42/200\n",
      "Epoch 42 - learning_rate 0.9995901680311072\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 0.9584 - acc: 0.6082 - lr: 0.9996 - val_loss: 0.9902 - val_acc: 0.5894 - val_lr: 0.9996\n",
      "Epoch 43/200\n",
      "Epoch 43 - learning_rate 0.999580176325943\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 0.9585 - acc: 0.6083 - lr: 0.9996 - val_loss: 0.9789 - val_acc: 0.5913 - val_lr: 0.9996\n",
      "Epoch 44/200\n",
      "Epoch 44 - learning_rate 0.9995701848205273\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9551 - acc: 0.6104 - lr: 0.9996 - val_loss: 0.9959 - val_acc: 0.5884 - val_lr: 0.9996\n",
      "Epoch 45/200\n",
      "Epoch 45 - learning_rate 0.9995601935148535\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9543 - acc: 0.6135 - lr: 0.9996 - val_loss: 0.9814 - val_acc: 0.5919 - val_lr: 0.9996\n",
      "Epoch 46/200\n",
      "Epoch 46 - learning_rate 0.9995502024089159\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9496 - acc: 0.6154 - lr: 0.9996 - val_loss: 0.9904 - val_acc: 0.5904 - val_lr: 0.9996\n",
      "Epoch 47/200\n",
      "Epoch 47 - learning_rate 0.9995402115027089\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9509 - acc: 0.6128 - lr: 0.9995 - val_loss: 0.9866 - val_acc: 0.5911 - val_lr: 0.9995\n",
      "Epoch 48/200\n",
      "Epoch 48 - learning_rate 0.9995302207962258\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.9478 - acc: 0.6165 - lr: 0.9995 - val_loss: 0.9772 - val_acc: 0.5885 - val_lr: 0.9995\n",
      "Epoch 49/200\n",
      "Epoch 49 - learning_rate 0.999520230289461\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9468 - acc: 0.6180 - lr: 0.9995 - val_loss: 0.9857 - val_acc: 0.5927 - val_lr: 0.9995\n",
      "Epoch 50/200\n",
      "Epoch 50 - learning_rate 0.9995102399824085\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9453 - acc: 0.6192 - lr: 0.9995 - val_loss: 0.9862 - val_acc: 0.5901 - val_lr: 0.9995\n",
      "Epoch 51/200\n",
      "Epoch 51 - learning_rate 0.9995002498750625\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.9468 - acc: 0.6172 - lr: 0.9995 - val_loss: 0.9819 - val_acc: 0.5939 - val_lr: 0.9995\n",
      "Epoch 52/200\n",
      "Epoch 52 - learning_rate 0.9994902599674166\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.9420 - acc: 0.6206 - lr: 0.9995 - val_loss: 0.9872 - val_acc: 0.5905 - val_lr: 0.9995\n",
      "Epoch 53/200\n",
      "Epoch 53 - learning_rate 0.999480270259465\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9412 - acc: 0.6171 - lr: 0.9995 - val_loss: 0.9807 - val_acc: 0.5896 - val_lr: 0.9995\n",
      "Epoch 54/200\n",
      "Epoch 54 - learning_rate 0.9994702807512019\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9370 - acc: 0.6196 - lr: 0.9995 - val_loss: 0.9877 - val_acc: 0.5886 - val_lr: 0.9995\n",
      "Epoch 55/200\n",
      "Epoch 55 - learning_rate 0.999460291442621\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.9387 - acc: 0.6195 - lr: 0.9995 - val_loss: 0.9801 - val_acc: 0.5899 - val_lr: 0.9995\n",
      "Epoch 56/200\n",
      "Epoch 56 - learning_rate 0.9994503023337165\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 0.9357 - acc: 0.6236 - lr: 0.9995 - val_loss: 0.9824 - val_acc: 0.5886 - val_lr: 0.9995\n",
      "Epoch 57/200\n",
      "Epoch 57 - learning_rate 0.9994403134244824\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.9337 - acc: 0.6230 - lr: 0.9994 - val_loss: 0.9900 - val_acc: 0.5889 - val_lr: 0.9994\n",
      "Epoch 58/200\n",
      "Epoch 58 - learning_rate 0.9994303247149126\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9339 - acc: 0.6236 - lr: 0.9994 - val_loss: 0.9779 - val_acc: 0.5890 - val_lr: 0.9994\n",
      "Epoch 59/200\n",
      "Epoch 59 - learning_rate 0.999420336205001\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.9311 - acc: 0.6246 - lr: 0.9994 - val_loss: 0.9853 - val_acc: 0.5885 - val_lr: 0.9994\n",
      "Epoch 60/200\n",
      "Epoch 60 - learning_rate 0.999410347894742\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9305 - acc: 0.6257 - lr: 0.9994 - val_loss: 0.9954 - val_acc: 0.5866 - val_lr: 0.9994\n",
      "Epoch 61/200\n",
      "Epoch 61 - learning_rate 0.9994003597841296\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9340 - acc: 0.6238 - lr: 0.9994 - val_loss: 0.9932 - val_acc: 0.5859 - val_lr: 0.9994\n",
      "Epoch 62/200\n",
      "Epoch 62 - learning_rate 0.9993903718731574\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9302 - acc: 0.6262 - lr: 0.9994 - val_loss: 0.9884 - val_acc: 0.5866 - val_lr: 0.9994\n",
      "Epoch 63/200\n",
      "Epoch 63 - learning_rate 0.9993803841618196\n",
      "32000/32000 [==============================] - 30s 943us/step - loss: 0.9297 - acc: 0.6268 - lr: 0.9994 - val_loss: 0.9885 - val_acc: 0.5895 - val_lr: 0.9994\n",
      "Epoch 64/200\n",
      "Epoch 64 - learning_rate 0.9993703966501105\n",
      "32000/32000 [==============================] - 32s 987us/step - loss: 0.9243 - acc: 0.6308 - lr: 0.9994 - val_loss: 0.9768 - val_acc: 0.5855 - val_lr: 0.9994\n",
      "Epoch 65/200\n",
      "Epoch 65 - learning_rate 0.9993604093380237\n",
      "32000/32000 [==============================] - 30s 927us/step - loss: 0.9268 - acc: 0.6269 - lr: 0.9994 - val_loss: 0.9855 - val_acc: 0.5869 - val_lr: 0.9994\n",
      "Epoch 66/200\n",
      "Epoch 66 - learning_rate 0.9993504222255534\n",
      "32000/32000 [==============================] - 29s 899us/step - loss: 0.9226 - acc: 0.6327 - lr: 0.9994 - val_loss: 0.9903 - val_acc: 0.5869 - val_lr: 0.9994\n",
      "Epoch 67/200\n",
      "Epoch 67 - learning_rate 9.993404353126936\n",
      "32000/32000 [==============================] - 32s 989us/step - loss: 0.9758 - acc: 0.6069 - lr: 9.9934 - val_loss: 1.0495 - val_acc: 0.5685 - val_lr: 9.9934\n",
      "Epoch 68/200\n",
      "Epoch 68 - learning_rate 0.9993304485994384\n",
      "32000/32000 [==============================] - 31s 979us/step - loss: 0.9342 - acc: 0.6297 - lr: 0.9993 - val_loss: 0.9980 - val_acc: 0.5866 - val_lr: 0.9993\n",
      "Epoch 69/200\n",
      "Epoch 69 - learning_rate 0.9993204620857816\n",
      "32000/32000 [==============================] - 29s 897us/step - loss: 0.9255 - acc: 0.6298 - lr: 0.9993 - val_loss: 0.9949 - val_acc: 0.5880 - val_lr: 0.9993\n",
      "Epoch 70/200\n",
      "Epoch 70 - learning_rate 0.9993104757717174\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000/32000 [==============================] - 29s 912us/step - loss: 0.9246 - acc: 0.6310 - lr: 0.9993 - val_loss: 0.9931 - val_acc: 0.5863 - val_lr: 0.9993\n",
      "Epoch 71/200\n",
      "Epoch 71 - learning_rate 0.99930048965724\n",
      "32000/32000 [==============================] - 31s 954us/step - loss: 0.9189 - acc: 0.6338 - lr: 0.9993 - val_loss: 0.9980 - val_acc: 0.5871 - val_lr: 0.9993\n",
      "Epoch 72/200\n",
      "Epoch 72 - learning_rate 0.999290503742343\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.9156 - acc: 0.6380 - lr: 0.9993 - val_loss: 0.9919 - val_acc: 0.5830 - val_lr: 0.9993\n",
      "Epoch 73/200\n",
      "Epoch 73 - learning_rate 0.9992805180270204\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.9137 - acc: 0.6363 - lr: 0.9993 - val_loss: 1.0009 - val_acc: 0.5841 - val_lr: 0.9993\n",
      "Epoch 74/200\n",
      "Epoch 74 - learning_rate 0.9992705325112669\n",
      "32000/32000 [==============================] - 27s 859us/step - loss: 0.9134 - acc: 0.6375 - lr: 0.9993 - val_loss: 0.9949 - val_acc: 0.5846 - val_lr: 0.9993\n",
      "Epoch 75/200\n",
      "Epoch 75 - learning_rate 0.9992605471950757\n",
      "32000/32000 [==============================] - 27s 854us/step - loss: 0.9088 - acc: 0.6398 - lr: 0.9993 - val_loss: 0.9919 - val_acc: 0.5827 - val_lr: 0.9993\n",
      "Epoch 76/200\n",
      "Epoch 76 - learning_rate 0.9992505620784411\n",
      "32000/32000 [==============================] - 27s 848us/step - loss: 0.9088 - acc: 0.6408 - lr: 0.9993 - val_loss: 1.0008 - val_acc: 0.5820 - val_lr: 0.9993\n",
      "Epoch 77/200\n",
      "Epoch 77 - learning_rate 0.9992405771613573\n",
      "32000/32000 [==============================] - 28s 886us/step - loss: 0.9058 - acc: 0.6395 - lr: 0.9992 - val_loss: 1.0052 - val_acc: 0.5837 - val_lr: 0.9992\n",
      "Epoch 78/200\n",
      "Epoch 78 - learning_rate 0.9992305924438183\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.9077 - acc: 0.6407 - lr: 0.9992 - val_loss: 0.9970 - val_acc: 0.5823 - val_lr: 0.9992\n",
      "Epoch 79/200\n",
      "Epoch 79 - learning_rate 0.9992206079258179\n",
      "32000/32000 [==============================] - 28s 861us/step - loss: 0.9049 - acc: 0.6424 - lr: 0.9992 - val_loss: 0.9893 - val_acc: 0.5825 - val_lr: 0.9992\n",
      "Epoch 80/200\n",
      "Epoch 80 - learning_rate 0.9992106236073501\n",
      "32000/32000 [==============================] - 27s 849us/step - loss: 0.9021 - acc: 0.6438 - lr: 0.9992 - val_loss: 1.0077 - val_acc: 0.5825 - val_lr: 0.9992\n",
      "Epoch 81/200\n",
      "Epoch 81 - learning_rate 0.9992006394884093\n",
      "32000/32000 [==============================] - 27s 856us/step - loss: 0.8980 - acc: 0.6458 - lr: 0.9992 - val_loss: 1.0019 - val_acc: 0.5843 - val_lr: 0.9992\n",
      "Epoch 82/200\n",
      "Epoch 82 - learning_rate 0.9991906555689891\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.8976 - acc: 0.6447 - lr: 0.9992 - val_loss: 1.0022 - val_acc: 0.5834 - val_lr: 0.9992\n",
      "Epoch 83/200\n",
      "Epoch 83 - learning_rate 0.9991806718490837\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.8974 - acc: 0.6482 - lr: 0.9992 - val_loss: 1.0050 - val_acc: 0.5857 - val_lr: 0.9992\n",
      "Epoch 84/200\n",
      "Epoch 84 - learning_rate 0.999170688328687\n",
      "32000/32000 [==============================] - 27s 856us/step - loss: 0.8925 - acc: 0.6490 - lr: 0.9992 - val_loss: 1.0078 - val_acc: 0.5805 - val_lr: 0.9992\n",
      "Epoch 85/200\n",
      "Epoch 85 - learning_rate 0.9991607050077935\n",
      "32000/32000 [==============================] - 27s 852us/step - loss: 0.8925 - acc: 0.6499 - lr: 0.9992 - val_loss: 0.9933 - val_acc: 0.5820 - val_lr: 0.9992\n",
      "Epoch 86/200\n",
      "Epoch 86 - learning_rate 0.9991507218863965\n",
      "32000/32000 [==============================] - 27s 844us/step - loss: 0.8941 - acc: 0.6458 - lr: 0.9992 - val_loss: 0.9959 - val_acc: 0.5794 - val_lr: 0.9992\n",
      "Epoch 87/200\n",
      "Epoch 87 - learning_rate 0.9991407389644904\n",
      "32000/32000 [==============================] - 28s 886us/step - loss: 0.8909 - acc: 0.6510 - lr: 0.9991 - val_loss: 0.9985 - val_acc: 0.5793 - val_lr: 0.9991\n",
      "Epoch 88/200\n",
      "Epoch 88 - learning_rate 0.9991307562420695\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.8899 - acc: 0.6482 - lr: 0.9991 - val_loss: 1.0047 - val_acc: 0.5815 - val_lr: 0.9991\n",
      "Epoch 89/200\n",
      "Epoch 89 - learning_rate 0.9991207737191272\n",
      "32000/32000 [==============================] - 28s 862us/step - loss: 0.8910 - acc: 0.6497 - lr: 0.9991 - val_loss: 1.0133 - val_acc: 0.5787 - val_lr: 0.9991\n",
      "Epoch 90/200\n",
      "Epoch 90 - learning_rate 0.9991107913956578\n",
      "32000/32000 [==============================] - 27s 849us/step - loss: 0.8868 - acc: 0.6514 - lr: 0.9991 - val_loss: 1.0016 - val_acc: 0.5791 - val_lr: 0.9991\n",
      "Epoch 91/200\n",
      "Epoch 91 - learning_rate 0.9991008092716556\n",
      "32000/32000 [==============================] - 28s 880us/step - loss: 0.8868 - acc: 0.6520 - lr: 0.9991 - val_loss: 0.9962 - val_acc: 0.5795 - val_lr: 0.9991\n",
      "Epoch 92/200\n",
      "Epoch 92 - learning_rate 0.9990908273471142\n",
      "32000/32000 [==============================] - 29s 895us/step - loss: 0.8873 - acc: 0.6517 - lr: 0.9991 - val_loss: 1.0124 - val_acc: 0.5783 - val_lr: 0.9991\n",
      "Epoch 93/200\n",
      "Epoch 93 - learning_rate 0.9990808456220277\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.8841 - acc: 0.6532 - lr: 0.9991 - val_loss: 1.0108 - val_acc: 0.5791 - val_lr: 0.9991\n",
      "Epoch 94/200\n",
      "Epoch 94 - learning_rate 0.9990708640963902\n",
      "32000/32000 [==============================] - 28s 860us/step - loss: 0.8853 - acc: 0.6525 - lr: 0.9991 - val_loss: 0.9972 - val_acc: 0.5777 - val_lr: 0.9991\n",
      "Epoch 95/200\n",
      "Epoch 95 - learning_rate 0.9990608827701961\n",
      "32000/32000 [==============================] - 27s 848us/step - loss: 0.8798 - acc: 0.6562 - lr: 0.9991 - val_loss: 1.0043 - val_acc: 0.5790 - val_lr: 0.9991\n",
      "Epoch 96/200\n",
      "Epoch 96 - learning_rate 0.9990509016434387\n",
      "32000/32000 [==============================] - 27s 851us/step - loss: 0.8816 - acc: 0.6557 - lr: 0.9991 - val_loss: 1.0055 - val_acc: 0.5831 - val_lr: 0.9991\n",
      "Epoch 97/200\n",
      "Epoch 97 - learning_rate 0.9990409207161125\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.8774 - acc: 0.6591 - lr: 0.9990 - val_loss: 1.0175 - val_acc: 0.5827 - val_lr: 0.9990\n",
      "Epoch 98/200\n",
      "Epoch 98 - learning_rate 0.9990309399882116\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 0.8736 - acc: 0.6597 - lr: 0.9990 - val_loss: 0.9984 - val_acc: 0.5790 - val_lr: 0.9990\n",
      "Epoch 99/200\n",
      "Epoch 99 - learning_rate 0.9990209594597295\n",
      "32000/32000 [==============================] - 28s 870us/step - loss: 0.8766 - acc: 0.6572 - lr: 0.9990 - val_loss: 1.0045 - val_acc: 0.5796 - val_lr: 0.9990\n",
      "Epoch 100/200\n",
      "Epoch 100 - learning_rate 9.990109791306606\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.9335 - acc: 0.6352 - lr: 9.9901 - val_loss: 1.0284 - val_acc: 0.5571 - val_lr: 9.9901\n",
      "Epoch 101/200\n",
      "Epoch 101 - learning_rate 0.9990009990009991\n",
      "32000/32000 [==============================] - 28s 875us/step - loss: 0.8918 - acc: 0.6531 - lr: 0.9990 - val_loss: 1.0116 - val_acc: 0.5829 - val_lr: 0.9990\n",
      "Epoch 102/200\n",
      "Epoch 102 - learning_rate 0.9989910190707386\n",
      "32000/32000 [==============================] - 28s 886us/step - loss: 0.8789 - acc: 0.6594 - lr: 0.9990 - val_loss: 1.0040 - val_acc: 0.5784 - val_lr: 0.9990\n",
      "Epoch 103/200\n",
      "Epoch 103 - learning_rate 0.9989810393398733\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.8704 - acc: 0.6620 - lr: 0.9990 - val_loss: 1.0245 - val_acc: 0.5815 - val_lr: 0.9990\n",
      "Epoch 104/200\n",
      "Epoch 104 - learning_rate 0.9989710598083973\n",
      "32000/32000 [==============================] - 28s 865us/step - loss: 0.8744 - acc: 0.6612 - lr: 0.9990 - val_loss: 1.0063 - val_acc: 0.5816 - val_lr: 0.9990\n",
      "Epoch 105/200\n",
      "Epoch 105 - learning_rate 0.9989610804763047\n",
      "32000/32000 [==============================] - 27s 850us/step - loss: 0.8725 - acc: 0.6613 - lr: 0.9990 - val_loss: 1.0038 - val_acc: 0.5795 - val_lr: 0.9990\n",
      "Epoch 106/200\n",
      "Epoch 106 - learning_rate 0.9989511013435892\n",
      "32000/32000 [==============================] - 27s 856us/step - loss: 0.8657 - acc: 0.6648 - lr: 0.9990 - val_loss: 1.0099 - val_acc: 0.5777 - val_lr: 0.9990\n",
      "Epoch 107/200\n",
      "Epoch 107 - learning_rate 0.9989411224102451\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.8642 - acc: 0.6670 - lr: 0.9989 - val_loss: 1.0106 - val_acc: 0.5803 - val_lr: 0.9989\n",
      "Epoch 108/200\n",
      "Epoch 108 - learning_rate 0.9989311436762665\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.8623 - acc: 0.6700 - lr: 0.9989 - val_loss: 1.0122 - val_acc: 0.5757 - val_lr: 0.9989\n",
      "Epoch 109/200\n",
      "Epoch 109 - learning_rate 0.9989211651416471\n",
      "32000/32000 [==============================] - 28s 864us/step - loss: 0.8603 - acc: 0.6670 - lr: 0.9989 - val_loss: 1.0226 - val_acc: 0.5760 - val_lr: 0.9989\n",
      "Epoch 110/200\n",
      "Epoch 110 - learning_rate 0.998911186806381\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.8550 - acc: 0.6687 - lr: 0.9989 - val_loss: 1.0303 - val_acc: 0.5733 - val_lr: 0.9989\n",
      "Epoch 111/200\n",
      "Epoch 111 - learning_rate 0.9989012086704624\n",
      "32000/32000 [==============================] - 28s 880us/step - loss: 0.8568 - acc: 0.6697 - lr: 0.9989 - val_loss: 1.0161 - val_acc: 0.5767 - val_lr: 0.9989\n",
      "Epoch 112/200\n",
      "Epoch 112 - learning_rate 0.9988912307338854\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.8545 - acc: 0.6692 - lr: 0.9989 - val_loss: 1.0173 - val_acc: 0.5755 - val_lr: 0.9989\n",
      "Epoch 113/200\n",
      "Epoch 113 - learning_rate 0.9988812529966438\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.8501 - acc: 0.6751 - lr: 0.9989 - val_loss: 1.0238 - val_acc: 0.5765 - val_lr: 0.9989\n",
      "Epoch 114/200\n",
      "Epoch 114 - learning_rate 0.9988712754587316\n",
      "32000/32000 [==============================] - 28s 871us/step - loss: 0.8519 - acc: 0.6719 - lr: 0.9989 - val_loss: 1.0233 - val_acc: 0.5764 - val_lr: 0.9989\n",
      "Epoch 115/200\n",
      "Epoch 115 - learning_rate 0.9988612981201431\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.8464 - acc: 0.6758 - lr: 0.9989 - val_loss: 1.0465 - val_acc: 0.5747 - val_lr: 0.9989\n",
      "Epoch 116/200\n",
      "Epoch 116 - learning_rate 0.998851320980872\n",
      "32000/32000 [==============================] - 28s 884us/step - loss: 0.8485 - acc: 0.6749 - lr: 0.9989 - val_loss: 1.0260 - val_acc: 0.5774 - val_lr: 0.9989\n",
      "Epoch 117/200\n",
      "Epoch 117 - learning_rate 0.9988413440409125\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 0.8470 - acc: 0.6737 - lr: 0.9988 - val_loss: 1.0306 - val_acc: 0.5800 - val_lr: 0.9988\n",
      "Epoch 118/200\n",
      "Epoch 118 - learning_rate 0.9988313673002588\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.8473 - acc: 0.6755 - lr: 0.9988 - val_loss: 1.0340 - val_acc: 0.5763 - val_lr: 0.9988\n",
      "Epoch 119/200\n",
      "Epoch 119 - learning_rate 0.9988213907589045\n",
      "32000/32000 [==============================] - 28s 867us/step - loss: 0.8451 - acc: 0.6758 - lr: 0.9988 - val_loss: 1.0161 - val_acc: 0.5760 - val_lr: 0.9988\n",
      "Epoch 120/200\n",
      "Epoch 120 - learning_rate 0.998811414416844\n",
      "32000/32000 [==============================] - 27s 855us/step - loss: 0.8428 - acc: 0.6768 - lr: 0.9988 - val_loss: 1.0220 - val_acc: 0.5751 - val_lr: 0.9988\n",
      "Epoch 121/200\n",
      "Epoch 121 - learning_rate 0.998801438274071\n",
      "32000/32000 [==============================] - 27s 853us/step - loss: 0.8423 - acc: 0.6783 - lr: 0.9988 - val_loss: 1.0220 - val_acc: 0.5753 - val_lr: 0.9988\n",
      "Epoch 122/200\n",
      "Epoch 122 - learning_rate 0.99879146233058\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.8388 - acc: 0.6798 - lr: 0.9988 - val_loss: 1.0296 - val_acc: 0.5756 - val_lr: 0.9988\n",
      "Epoch 123/200\n",
      "Epoch 123 - learning_rate 0.9987814865863647\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 0.8383 - acc: 0.6788 - lr: 0.9988 - val_loss: 1.0216 - val_acc: 0.5756 - val_lr: 0.9988\n",
      "Epoch 124/200\n",
      "Epoch 124 - learning_rate 0.998771511041419\n",
      "32000/32000 [==============================] - 28s 868us/step - loss: 0.8359 - acc: 0.6804 - lr: 0.9988 - val_loss: 1.0270 - val_acc: 0.5761 - val_lr: 0.9988\n",
      "Epoch 125/200\n",
      "Epoch 125 - learning_rate 0.9987615356957373\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.8360 - acc: 0.6798 - lr: 0.9988 - val_loss: 1.0265 - val_acc: 0.5729 - val_lr: 0.9988\n",
      "Epoch 126/200\n",
      "Epoch 126 - learning_rate 0.9987515605493134\n",
      "32000/32000 [==============================] - 28s 881us/step - loss: 0.8309 - acc: 0.6824 - lr: 0.9988 - val_loss: 1.0257 - val_acc: 0.5735 - val_lr: 0.9988\n",
      "Epoch 127/200\n",
      "Epoch 127 - learning_rate 0.9987415856021412\n",
      "32000/32000 [==============================] - 28s 885us/step - loss: 0.8309 - acc: 0.6807 - lr: 0.9987 - val_loss: 1.0228 - val_acc: 0.5750 - val_lr: 0.9987\n",
      "Epoch 128/200\n",
      "Epoch 128 - learning_rate 0.998731610854215\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.8268 - acc: 0.6842 - lr: 0.9987 - val_loss: 1.0187 - val_acc: 0.5737 - val_lr: 0.9987\n",
      "Epoch 129/200\n",
      "Epoch 129 - learning_rate 0.998721636305529\n",
      "32000/32000 [==============================] - 28s 873us/step - loss: 0.8292 - acc: 0.6844 - lr: 0.9987 - val_loss: 1.0292 - val_acc: 0.5696 - val_lr: 0.9987\n",
      "Epoch 130/200\n",
      "Epoch 130 - learning_rate 0.9987116619560766\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.8274 - acc: 0.6856 - lr: 0.9987 - val_loss: 1.0466 - val_acc: 0.5730 - val_lr: 0.9987\n",
      "Epoch 131/200\n",
      "Epoch 131 - learning_rate 0.9987016878058523\n",
      "32000/32000 [==============================] - 28s 882us/step - loss: 0.8272 - acc: 0.6848 - lr: 0.9987 - val_loss: 1.0364 - val_acc: 0.5740 - val_lr: 0.9987\n",
      "Epoch 132/200\n",
      "Epoch 132 - learning_rate 0.9986917138548502\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.8258 - acc: 0.6859 - lr: 0.9987 - val_loss: 1.0294 - val_acc: 0.5759 - val_lr: 0.9987\n",
      "Epoch 133/200\n",
      "Epoch 133 - learning_rate 9.98681740103064\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.8913 - acc: 0.6589 - lr: 9.9868 - val_loss: 1.0548 - val_acc: 0.5670 - val_lr: 9.9868\n",
      "Epoch 134/200\n",
      "Epoch 134 - learning_rate 0.9986717665504878\n",
      "32000/32000 [==============================] - 28s 867us/step - loss: 0.8444 - acc: 0.6810 - lr: 0.9987 - val_loss: 1.0387 - val_acc: 0.5735 - val_lr: 0.9987\n",
      "Epoch 135/200\n",
      "Epoch 135 - learning_rate 0.998661793197116\n",
      "32000/32000 [==============================] - 27s 849us/step - loss: 0.8314 - acc: 0.6843 - lr: 0.9987 - val_loss: 1.0382 - val_acc: 0.5723 - val_lr: 0.9987\n",
      "Epoch 136/200\n",
      "Epoch 136 - learning_rate 0.9986518200429421\n",
      "32000/32000 [==============================] - 27s 854us/step - loss: 0.8248 - acc: 0.6891 - lr: 0.9987 - val_loss: 1.0354 - val_acc: 0.5706 - val_lr: 0.9987\n",
      "Epoch 137/200\n",
      "Epoch 137 - learning_rate 0.9986418470879603\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.8215 - acc: 0.6931 - lr: 0.9986 - val_loss: 1.0362 - val_acc: 0.5717 - val_lr: 0.9986\n",
      "Epoch 138/200\n",
      "Epoch 138 - learning_rate 0.9986318743321648\n",
      "32000/32000 [==============================] - 29s 894us/step - loss: 0.8163 - acc: 0.6921 - lr: 0.9986 - val_loss: 1.0462 - val_acc: 0.5709 - val_lr: 0.9986\n",
      "Epoch 139/200\n",
      "Epoch 139 - learning_rate 0.9986219017755498\n",
      "32000/32000 [==============================] - 28s 868us/step - loss: 0.8143 - acc: 0.6913 - lr: 0.9986 - val_loss: 1.0346 - val_acc: 0.5717 - val_lr: 0.9986\n",
      "Epoch 140/200\n",
      "Epoch 140 - learning_rate 0.9986119294181088\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.8100 - acc: 0.6940 - lr: 0.9986 - val_loss: 1.0354 - val_acc: 0.5711 - val_lr: 0.9986\n",
      "Epoch 141/200\n",
      "Epoch 141 - learning_rate 0.9986019572598361\n",
      "32000/32000 [==============================] - 28s 885us/step - loss: 0.8116 - acc: 0.6963 - lr: 0.9986 - val_loss: 1.0373 - val_acc: 0.5725 - val_lr: 0.9986\n",
      "Epoch 142/200\n",
      "Epoch 142 - learning_rate 0.998591985300726\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.8082 - acc: 0.6995 - lr: 0.9986 - val_loss: 1.0379 - val_acc: 0.5700 - val_lr: 0.9986\n",
      "Epoch 143/200\n",
      "Epoch 143 - learning_rate 0.9985820135407721\n",
      "32000/32000 [==============================] - 28s 885us/step - loss: 0.8075 - acc: 0.6962 - lr: 0.9986 - val_loss: 1.0441 - val_acc: 0.5731 - val_lr: 0.9986\n",
      "Epoch 144/200\n",
      "Epoch 144 - learning_rate 0.9985720419799686\n",
      "32000/32000 [==============================] - 28s 874us/step - loss: 0.8074 - acc: 0.6969 - lr: 0.9986 - val_loss: 1.0317 - val_acc: 0.5740 - val_lr: 0.9986\n",
      "Epoch 145/200\n",
      "Epoch 145 - learning_rate 0.9985620706183095\n",
      "32000/32000 [==============================] - 27s 846us/step - loss: 0.8051 - acc: 0.6976 - lr: 0.9986 - val_loss: 1.0469 - val_acc: 0.5720 - val_lr: 0.9986\n",
      "Epoch 146/200\n",
      "Epoch 146 - learning_rate 0.9985520994557892\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000/32000 [==============================] - 28s 885us/step - loss: 0.7986 - acc: 0.7019 - lr: 0.9986 - val_loss: 1.0421 - val_acc: 0.5736 - val_lr: 0.9986\n",
      "Epoch 147/200\n",
      "Epoch 147 - learning_rate 0.9985421284924011\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 0.8006 - acc: 0.6994 - lr: 0.9985 - val_loss: 1.0541 - val_acc: 0.5686 - val_lr: 0.9985\n",
      "Epoch 148/200\n",
      "Epoch 148 - learning_rate 0.9985321577281395\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.7967 - acc: 0.7025 - lr: 0.9985 - val_loss: 1.0590 - val_acc: 0.5691 - val_lr: 0.9985\n",
      "Epoch 149/200\n",
      "Epoch 149 - learning_rate 0.9985221871629988\n",
      "32000/32000 [==============================] - 28s 863us/step - loss: 0.7986 - acc: 0.7020 - lr: 0.9985 - val_loss: 1.0390 - val_acc: 0.5683 - val_lr: 0.9985\n",
      "Epoch 150/200\n",
      "Epoch 150 - learning_rate 0.9985122167969726\n",
      "32000/32000 [==============================] - 27s 848us/step - loss: 0.7946 - acc: 0.7035 - lr: 0.9985 - val_loss: 1.0473 - val_acc: 0.5726 - val_lr: 0.9985\n",
      "Epoch 151/200\n",
      "Epoch 151 - learning_rate 0.9985022466300548\n",
      "32000/32000 [==============================] - 28s 880us/step - loss: 0.7917 - acc: 0.7060 - lr: 0.9985 - val_loss: 1.0478 - val_acc: 0.5711 - val_lr: 0.9985\n",
      "Epoch 152/200\n",
      "Epoch 152 - learning_rate 0.9984922766622402\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 0.7924 - acc: 0.7046 - lr: 0.9985 - val_loss: 1.0501 - val_acc: 0.5680 - val_lr: 0.9985\n",
      "Epoch 153/200\n",
      "Epoch 153 - learning_rate 0.9984823068935219\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 0.7875 - acc: 0.7063 - lr: 0.9985 - val_loss: 1.0660 - val_acc: 0.5640 - val_lr: 0.9985\n",
      "Epoch 154/200\n",
      "Epoch 154 - learning_rate 0.9984723373238944\n",
      "32000/32000 [==============================] - 28s 872us/step - loss: 0.7860 - acc: 0.7058 - lr: 0.9985 - val_loss: 1.0787 - val_acc: 0.5611 - val_lr: 0.9985\n",
      "Epoch 155/200\n",
      "Epoch 155 - learning_rate 0.9984623679533517\n",
      "32000/32000 [==============================] - 27s 851us/step - loss: 0.7835 - acc: 0.7087 - lr: 0.9985 - val_loss: 1.0535 - val_acc: 0.5690 - val_lr: 0.9985\n",
      "Epoch 156/200\n",
      "Epoch 156 - learning_rate 0.9984523987818882\n",
      "32000/32000 [==============================] - 28s 863us/step - loss: 0.7867 - acc: 0.7071 - lr: 0.9985 - val_loss: 1.0370 - val_acc: 0.5699 - val_lr: 0.9985\n",
      "Epoch 157/200\n",
      "Epoch 157 - learning_rate 0.9984424298094972\n",
      "32000/32000 [==============================] - 29s 893us/step - loss: 0.7814 - acc: 0.7097 - lr: 0.9984 - val_loss: 1.0411 - val_acc: 0.5674 - val_lr: 0.9984\n",
      "Epoch 158/200\n",
      "Epoch 158 - learning_rate 0.9984324610361731\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.7811 - acc: 0.7100 - lr: 0.9984 - val_loss: 1.0623 - val_acc: 0.5701 - val_lr: 0.9984\n",
      "Epoch 159/200\n",
      "Epoch 159 - learning_rate 0.9984224924619103\n",
      "32000/32000 [==============================] - 28s 870us/step - loss: 0.7797 - acc: 0.7110 - lr: 0.9984 - val_loss: 1.0515 - val_acc: 0.5663 - val_lr: 0.9984\n",
      "Epoch 160/200\n",
      "Epoch 160 - learning_rate 0.9984125240867021\n",
      "32000/32000 [==============================] - 27s 850us/step - loss: 0.7770 - acc: 0.7103 - lr: 0.9984 - val_loss: 1.0519 - val_acc: 0.5649 - val_lr: 0.9984\n",
      "Epoch 161/200\n",
      "Epoch 161 - learning_rate 0.998402555910543\n",
      "32000/32000 [==============================] - 27s 852us/step - loss: 0.7776 - acc: 0.7106 - lr: 0.9984 - val_loss: 1.0513 - val_acc: 0.5675 - val_lr: 0.9984\n",
      "Epoch 162/200\n",
      "Epoch 162 - learning_rate 0.9983925879334273\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.7746 - acc: 0.7147 - lr: 0.9984 - val_loss: 1.0568 - val_acc: 0.5639 - val_lr: 0.9984\n",
      "Epoch 163/200\n",
      "Epoch 163 - learning_rate 0.9983826201553484\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.7711 - acc: 0.7148 - lr: 0.9984 - val_loss: 1.0549 - val_acc: 0.5649 - val_lr: 0.9984\n",
      "Epoch 164/200\n",
      "Epoch 164 - learning_rate 0.9983726525763006\n",
      "32000/32000 [==============================] - 28s 862us/step - loss: 0.7707 - acc: 0.7165 - lr: 0.9984 - val_loss: 1.0475 - val_acc: 0.5669 - val_lr: 0.9984\n",
      "Epoch 165/200\n",
      "Epoch 165 - learning_rate 0.998362685196278\n",
      "32000/32000 [==============================] - 27s 851us/step - loss: 0.7693 - acc: 0.7165 - lr: 0.9984 - val_loss: 1.0790 - val_acc: 0.5600 - val_lr: 0.9984\n",
      "Epoch 166/200\n",
      "Epoch 166 - learning_rate 9.983527180152748\n",
      "32000/32000 [==============================] - 27s 850us/step - loss: 0.8400 - acc: 0.6888 - lr: 9.9835 - val_loss: 1.0678 - val_acc: 0.5640 - val_lr: 9.9835\n",
      "Epoch 167/200\n",
      "Epoch 167 - learning_rate 0.9983427510332847\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.7919 - acc: 0.7089 - lr: 0.9983 - val_loss: 1.0697 - val_acc: 0.5654 - val_lr: 0.9983\n",
      "Epoch 168/200\n",
      "Epoch 168 - learning_rate 0.9983327842503019\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.7752 - acc: 0.7136 - lr: 0.9983 - val_loss: 1.0751 - val_acc: 0.5625 - val_lr: 0.9983\n",
      "Epoch 169/200\n",
      "Epoch 169 - learning_rate 0.9983228176663207\n",
      "32000/32000 [==============================] - 28s 873us/step - loss: 0.7692 - acc: 0.7169 - lr: 0.9983 - val_loss: 1.0761 - val_acc: 0.5620 - val_lr: 0.9983\n",
      "Epoch 170/200\n",
      "Epoch 170 - learning_rate 0.9983128512813346\n",
      "32000/32000 [==============================] - 27s 854us/step - loss: 0.7654 - acc: 0.7189 - lr: 0.9983 - val_loss: 1.0776 - val_acc: 0.5629 - val_lr: 0.9983\n",
      "Epoch 171/200\n",
      "Epoch 171 - learning_rate 0.9983028850953379\n",
      "32000/32000 [==============================] - 28s 878us/step - loss: 0.7633 - acc: 0.7198 - lr: 0.9983 - val_loss: 1.0765 - val_acc: 0.5649 - val_lr: 0.9983\n",
      "Epoch 172/200\n",
      "Epoch 172 - learning_rate 0.9982929191083246\n",
      "32000/32000 [==============================] - 28s 887us/step - loss: 0.7639 - acc: 0.7169 - lr: 0.9983 - val_loss: 1.0692 - val_acc: 0.5646 - val_lr: 0.9983\n",
      "Epoch 173/200\n",
      "Epoch 173 - learning_rate 0.9982829533202892\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.7568 - acc: 0.7215 - lr: 0.9983 - val_loss: 1.0602 - val_acc: 0.5651 - val_lr: 0.9983\n",
      "Epoch 174/200\n",
      "Epoch 174 - learning_rate 0.998272987731225\n",
      "32000/32000 [==============================] - 28s 866us/step - loss: 0.7572 - acc: 0.7221 - lr: 0.9983 - val_loss: 1.0758 - val_acc: 0.5617 - val_lr: 0.9983\n",
      "Epoch 175/200\n",
      "Epoch 175 - learning_rate 0.9982630223411264\n",
      "32000/32000 [==============================] - 27s 848us/step - loss: 0.7539 - acc: 0.7255 - lr: 0.9983 - val_loss: 1.0745 - val_acc: 0.5639 - val_lr: 0.9983\n",
      "Epoch 176/200\n",
      "Epoch 176 - learning_rate 0.9982530571499876\n",
      "32000/32000 [==============================] - 27s 856us/step - loss: 0.7556 - acc: 0.7251 - lr: 0.9983 - val_loss: 1.0744 - val_acc: 0.5630 - val_lr: 0.9983\n",
      "Epoch 177/200\n",
      "Epoch 177 - learning_rate 0.9982430921578023\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.7494 - acc: 0.7271 - lr: 0.9982 - val_loss: 1.0758 - val_acc: 0.5634 - val_lr: 0.9982\n",
      "Epoch 178/200\n",
      "Epoch 178 - learning_rate 0.9982331273645647\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.7487 - acc: 0.7273 - lr: 0.9982 - val_loss: 1.0776 - val_acc: 0.5639 - val_lr: 0.9982\n",
      "Epoch 179/200\n",
      "Epoch 179 - learning_rate 0.998223162770269\n",
      "32000/32000 [==============================] - 28s 866us/step - loss: 0.7487 - acc: 0.7270 - lr: 0.9982 - val_loss: 1.0751 - val_acc: 0.5627 - val_lr: 0.9982\n",
      "Epoch 180/200\n",
      "Epoch 180 - learning_rate 0.998213198374909\n",
      "32000/32000 [==============================] - 27s 858us/step - loss: 0.7442 - acc: 0.7273 - lr: 0.9982 - val_loss: 1.0689 - val_acc: 0.5654 - val_lr: 0.9982\n",
      "Epoch 181/200\n",
      "Epoch 181 - learning_rate 0.9982032341784787\n",
      "32000/32000 [==============================] - 27s 854us/step - loss: 0.7445 - acc: 0.7272 - lr: 0.9982 - val_loss: 1.0836 - val_acc: 0.5613 - val_lr: 0.9982\n",
      "Epoch 182/200\n",
      "Epoch 182 - learning_rate 0.9981932701809724\n",
      "32000/32000 [==============================] - 29s 892us/step - loss: 0.7386 - acc: 0.7312 - lr: 0.9982 - val_loss: 1.0768 - val_acc: 0.5646 - val_lr: 0.9982\n",
      "Epoch 183/200\n",
      "Epoch 183 - learning_rate 0.9981833063823842\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.7430 - acc: 0.7291 - lr: 0.9982 - val_loss: 1.0735 - val_acc: 0.5620 - val_lr: 0.9982\n",
      "Epoch 184/200\n",
      "Epoch 184 - learning_rate 0.9981733427827076\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000/32000 [==============================] - 28s 883us/step - loss: 0.7375 - acc: 0.7320 - lr: 0.9982 - val_loss: 1.0751 - val_acc: 0.5633 - val_lr: 0.9982\n",
      "Epoch 185/200\n",
      "Epoch 185 - learning_rate 0.9981633793819372\n",
      "32000/32000 [==============================] - 27s 854us/step - loss: 0.7394 - acc: 0.7323 - lr: 0.9982 - val_loss: 1.0780 - val_acc: 0.5601 - val_lr: 0.9982\n",
      "Epoch 186/200\n",
      "Epoch 186 - learning_rate 0.9981534161800669\n",
      "32000/32000 [==============================] - 27s 846us/step - loss: 0.7354 - acc: 0.7317 - lr: 0.9982 - val_loss: 1.0920 - val_acc: 0.5594 - val_lr: 0.9982\n",
      "Epoch 187/200\n",
      "Epoch 187 - learning_rate 0.9981434531770906\n",
      "32000/32000 [==============================] - 28s 886us/step - loss: 0.7331 - acc: 0.7324 - lr: 0.9981 - val_loss: 1.0936 - val_acc: 0.5607 - val_lr: 0.9981\n",
      "Epoch 188/200\n",
      "Epoch 188 - learning_rate 0.9981334903730025\n",
      "32000/32000 [==============================] - 28s 888us/step - loss: 0.7337 - acc: 0.7348 - lr: 0.9981 - val_loss: 1.0921 - val_acc: 0.5599 - val_lr: 0.9981\n",
      "Epoch 189/200\n",
      "Epoch 189 - learning_rate 0.9981235277677964\n",
      "32000/32000 [==============================] - 28s 868us/step - loss: 0.7310 - acc: 0.7359 - lr: 0.9981 - val_loss: 1.0864 - val_acc: 0.5597 - val_lr: 0.9981\n",
      "Epoch 190/200\n",
      "Epoch 190 - learning_rate 0.9981135653614669\n",
      "32000/32000 [==============================] - 27s 853us/step - loss: 0.7305 - acc: 0.7348 - lr: 0.9981 - val_loss: 1.0882 - val_acc: 0.5605 - val_lr: 0.9981\n",
      "Epoch 191/200\n",
      "Epoch 191 - learning_rate 0.9981036031540074\n",
      "32000/32000 [==============================] - 27s 843us/step - loss: 0.7279 - acc: 0.7359 - lr: 0.9981 - val_loss: 1.0972 - val_acc: 0.5597 - val_lr: 0.9981\n",
      "Epoch 192/200\n",
      "Epoch 192 - learning_rate 0.9980936411454122\n",
      "32000/32000 [==============================] - 29s 891us/step - loss: 0.7241 - acc: 0.7375 - lr: 0.9981 - val_loss: 1.0932 - val_acc: 0.5610 - val_lr: 0.9981\n",
      "Epoch 193/200\n",
      "Epoch 193 - learning_rate 0.9980836793356755\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.7221 - acc: 0.7372 - lr: 0.9981 - val_loss: 1.0995 - val_acc: 0.5574 - val_lr: 0.9981\n",
      "Epoch 194/200\n",
      "Epoch 194 - learning_rate 0.9980737177247911\n",
      "32000/32000 [==============================] - 28s 870us/step - loss: 0.7212 - acc: 0.7380 - lr: 0.9981 - val_loss: 1.0916 - val_acc: 0.5581 - val_lr: 0.9981\n",
      "Epoch 195/200\n",
      "Epoch 195 - learning_rate 0.9980637563127532\n",
      "32000/32000 [==============================] - 27s 857us/step - loss: 0.7229 - acc: 0.7375 - lr: 0.9981 - val_loss: 1.0959 - val_acc: 0.5581 - val_lr: 0.9981\n",
      "Epoch 196/200\n",
      "Epoch 196 - learning_rate 0.9980537950995559\n",
      "32000/32000 [==============================] - 27s 854us/step - loss: 0.7228 - acc: 0.7365 - lr: 0.9981 - val_loss: 1.0882 - val_acc: 0.5589 - val_lr: 0.9981\n",
      "Epoch 197/200\n",
      "Epoch 197 - learning_rate 0.998043834085193\n",
      "32000/32000 [==============================] - 28s 889us/step - loss: 0.7200 - acc: 0.7398 - lr: 0.9980 - val_loss: 1.0905 - val_acc: 0.5605 - val_lr: 0.9980\n",
      "Epoch 198/200\n",
      "Epoch 198 - learning_rate 0.9980338732696588\n",
      "32000/32000 [==============================] - 28s 890us/step - loss: 0.7151 - acc: 0.7412 - lr: 0.9980 - val_loss: 1.0891 - val_acc: 0.5611 - val_lr: 0.9980\n",
      "Epoch 199/200\n",
      "Epoch 199 - learning_rate 9.98023912652947\n",
      "32000/32000 [==============================] - 28s 861us/step - loss: 0.7918 - acc: 0.7152 - lr: 9.9802 - val_loss: 1.1081 - val_acc: 0.5474 - val_lr: 9.9802\n",
      "Epoch 200/200\n",
      "Epoch 200 - learning_rate 0.9980139522350523\n",
      "32000/32000 [==============================] - 27s 847us/step - loss: 0.7390 - acc: 0.7351 - lr: 0.9980 - val_loss: 1.1189 - val_acc: 0.5543 - val_lr: 0.9980\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x149b9bf28>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.summary()\n",
    "print(\"Training Progress:\")\n",
    "model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          epochs=200, batch_size=64,\n",
    "          callbacks=[tensorboard, model_checkpoints, lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('ltsm-c.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
